{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "authorship_tag": "ABX9TyPt/CKiHtiY/0SQF9VBw4xf",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ktynski/Marketing_Automations_Notebooks_With_GPT/blob/main/Automatic_Content_Gap_Report_and_Analysis_With_Clustering_and_Cluster_Descriptions_(Public).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install openai\n",
        "!pip install transformers\n",
        "!pip install kneed"
      ],
      "metadata": {
        "id": "fbTsKQZCJcPW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0BqpW3sPJU3t"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "import pandas as pd\n",
        "import openai\n",
        "import csv\n",
        "from io import StringIO\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "from sklearn.cluster import KMeans\n",
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "from kneed import KneeLocator\n",
        "from transformers import GPT2Tokenizer\n",
        "\n",
        "\n",
        "# Define API keys and endpoints\n",
        "SEMRUSH_API_KEY = \"Your SEMRush API Key\"\n",
        "openai.api_key = \"Your Openai API Key\"\n",
        "SEMRUSH_API_ENDPOINT = \"https://api.semrush.com/\"\n",
        "\n",
        "\n",
        "\n",
        "def analyze_user_website_content(domain, min_difficulty, max_difficulty, num_keywords, min_position, max_position):\n",
        "    api_endpoint = \"https://api.semrush.com/\"\n",
        "    params = {\n",
        "        \"type\": \"domain_organic\",\n",
        "        \"key\": SEMRUSH_API_KEY,\n",
        "        \"domain\": domain,\n",
        "        \"export_columns\": \"Ph,Kd,Po,Ps\",  # Include \"Position\" column\n",
        "        \"database\": \"us\",\n",
        "        \"display_limit\": num_keywords,\n",
        "        \"display_filter\": f\"+|Kd|Bt|{min_difficulty}|{max_difficulty};+|Ps|Bt|{min_position}|{max_position}\"\n",
        "    }\n",
        "    response = requests.get(api_endpoint, params=params)\n",
        "    csv_data = response.text\n",
        "    print(csv_data)\n",
        "    df = pd.read_csv(StringIO(csv_data), delimiter=';', names=[\"Keyword\", \"Keyword Difficulty\", \"Position\"])\n",
        "    # Convert the \"Keyword Difficulty\" and \"Position\" columns to float\n",
        "    df[\"Keyword Difficulty\"] = pd.to_numeric(df[\"Keyword Difficulty\"], errors='coerce')\n",
        "    df[\"Position\"] = pd.to_numeric(df[\"Position\"], errors='coerce')\n",
        "    # Filter based on keyword difficulty and position range\n",
        "    df_filtered = df[(df[\"Keyword Difficulty\"] >= float(min_difficulty)) &\n",
        "                     (df[\"Keyword Difficulty\"] <= float(max_difficulty)) &\n",
        "                     (df[\"Position\"] >= float(min_position)) &\n",
        "                     (df[\"Position\"] <= float(max_position))]\n",
        "\n",
        "    return df_filtered\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def retrieve_top_ranking_websites(topic_area, num_websites):\n",
        "    api_endpoint = \"https://api.semrush.com/\"\n",
        "    params = {\n",
        "        \"type\": \"phrase_organic\",\n",
        "        \"key\": SEMRUSH_API_KEY,\n",
        "        \"phrase\": topic_area,\n",
        "        \"export_columns\": \"Dn\",\n",
        "        \"database\": \"us\",\n",
        "        \"display_limit\": num_websites\n",
        "    }\n",
        "    response = requests.get(api_endpoint, params=params)\n",
        "    csv_data = response.text\n",
        "    #print(csv_data)\n",
        "    df = pd.read_csv(StringIO(csv_data), skiprows=1, names=[\"Domain\"])\n",
        "    #print(\"Top-ranking competitor domains:\")\n",
        "    #print(df)\n",
        "    return df\n",
        "\n",
        "def identify_content_gaps(user_domain, topic_area, min_difficulty, max_difficulty, num_websites, num_keywords, user_position_range, competitor_position_range):\n",
        "    user_min_position, user_max_position = user_position_range\n",
        "    user_keywords_df = analyze_user_website_content(user_domain, min_difficulty, max_difficulty, num_keywords, user_min_position, user_max_position)\n",
        "    user_keywords_df.to_csv('Keywords_from_user_domain.csv')\n",
        "    competitor_domains_df = retrieve_top_ranking_websites(topic_area, num_websites)\n",
        "    competitor_domains_df.to_csv('Keywords_from_competitor_domains.csv')\n",
        "    competitor_keywords = set()\n",
        "    competitor_min_position, competitor_max_position = competitor_position_range\n",
        "    for _, row in competitor_domains_df.iterrows():\n",
        "        domain_keywords_df = analyze_user_website_content(row[\"Domain\"], min_difficulty, max_difficulty, num_keywords, competitor_min_position, competitor_max_position)\n",
        "        competitor_keywords.update(domain_keywords_df[\"Keyword\"].tolist())\n",
        "    #print(competitor_keywords)\n",
        "    content_gaps = competitor_keywords - set(user_keywords_df[\"Keyword\"].tolist())\n",
        "    #print(\"Content Gaps:\")\n",
        "    #print(content_gaps)\n",
        "    return content_gaps\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def generate_embeddings(keywords, model_name=\"sentence-transformers/bert-base-nli-mean-tokens\"):\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "    model = AutoModel.from_pretrained(model_name)\n",
        "    embeddings = []\n",
        "    for keyword in keywords:\n",
        "        inputs = tokenizer(keyword, return_tensors=\"pt\", padding=True, truncation=True)\n",
        "        with torch.no_grad():\n",
        "            outputs = model(**inputs)\n",
        "            embeddings.append(outputs.last_hidden_state.mean(dim=1).squeeze().numpy())\n",
        "    return embeddings\n",
        "\n",
        "def perform_clustering(embeddings):\n",
        "    cluster_range = range(10, 40)\n",
        "    inertia_values = []\n",
        "    for n_clusters in cluster_range:\n",
        "        kmeans = KMeans(n_clusters=n_clusters, random_state=0)\n",
        "        kmeans.fit(embeddings)\n",
        "        inertia_values.append(kmeans.inertia_)\n",
        "\n",
        "    # Use the KneeLocator to automatically detect the elbow point\n",
        "    knee_locator = KneeLocator(cluster_range, inertia_values, curve=\"convex\", direction=\"decreasing\", S=0.3)\n",
        "    n_clusters_optimal = knee_locator.elbow\n",
        "\n",
        "    # Perform clustering with the optimal number of clusters\n",
        "    kmeans_optimal = KMeans(n_clusters=n_clusters_optimal, random_state=0).fit(embeddings)\n",
        "    return kmeans_optimal.labels_\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def describe_clusters(clustered_keywords_df):\n",
        "    cluster_descriptions = {}\n",
        "    for cluster_id, group in clustered_keywords_df.groupby('Cluster'):\n",
        "        # Get the keywords for the current cluster\n",
        "        keywords = group['Keyword'].tolist()\n",
        "\n",
        "        # Convert the keywords into a comma-separated string\n",
        "        keywords_str = ', '.join(keywords)\n",
        "\n",
        "        # Set the instructions for GPT-3\n",
        "        instructions = f\"Describe the topics related to the following keywords: {keywords_str}\"\n",
        "\n",
        "        # Make an API call to GPT-3\n",
        "        response = openai.ChatCompletion.create(\n",
        "            model=\"gpt-3.5-turbo\",\n",
        "            messages=[\n",
        "                {\"role\": \"system\", \"content\": \"You are an AI language model.\"},\n",
        "                {\"role\": \"assistant\", \"content\": \"Provide a description for the given cluster. It is possible that some clusters have keywords that dont really seem to fit. Your job is to find the primary themes and disregard keywords that dont seem to fit and generate the description based on the keywords that most fit.\"},\n",
        "                {\"role\": \"user\", \"content\": instructions}\n",
        "            ],\n",
        "            temperature=.5,\n",
        "            max_tokens=256,\n",
        "            top_p=1,\n",
        "            frequency_penalty=0,\n",
        "            presence_penalty=0\n",
        "        )\n",
        "\n",
        "        # Extract the description from the GPT-3 response\n",
        "        description = response[\"choices\"][0][\"message\"][\"content\"].strip()\n",
        "\n",
        "        # Store the description in the dictionary\n",
        "        cluster_descriptions[cluster_id] = description\n",
        "\n",
        "    return cluster_descriptions\n",
        "\n",
        "def create_clustered_keywords_df(keywords, cluster_assignments):\n",
        "    # Create a DataFrame with the keywords and their corresponding cluster assignments\n",
        "    df = pd.DataFrame({'Keyword': keywords, 'Cluster': cluster_assignments})\n",
        "    return df\n",
        "\n",
        "\n",
        "def generate_cluster_analysis_report(keywords,user_domain):\n",
        "    # Prepare the input message for GPT-3\n",
        "\n",
        "    input_message = f\"You are an expert SEO analyst with extensive experience in analyzing keyword groups and making recommendations based on keyword gap analysis data which will be provided. and creating highly informative and actionable SEO keyword gap and content strategy reports for clients. I have conducted a content gap analysis for a brand and identified a list of keywords that represent potential content opportunities. I would like you to provide an overall analysis and breakdown of the keyword clusters based on their topical and semantic relevance. Please create a comprehensive SEO report that includes the following sections:\\n 1. Introduction: Provide an overview of the content gap analysis and its importance for the brand.\\n 2. Keyword Clusters: Identify and describe the main topical clusters present in the list of keywords. Give a few representative keywords but dont list too many. Explain the relevance and potential impact of each cluster for the brand's SEO strategy.\\n 3. Recommendations: Provide specific content creation and optimization recommendations for each keyword cluster. Explain how these recommendations can help the brand improve its search visibility and attract more organic traffic.\\n 4. Conclusion: Summarize the key findings and recommendations of the report. And give any/all additional thoughts that might be useful or helpful based on the learnings from the gap analysis.\\n Please present the information in a clear and professional manner, as though it were a client SEO report. Please also at the end provide a list of 25 article idea titles that would help most in improving where there are gaps and also would make useful and interesting articles that spark curiosity. Here is the list of keywords from the content gap analysis:\\n {', '.join(keywords)}\\n\"\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    # Make the GPT-3 API call\n",
        "    response = openai.ChatCompletion.create(\n",
        "        model=\"gpt-4\",\n",
        "        messages=[\n",
        "            {\"role\": \"system\", \"content\": input_message},\n",
        "            {\"role\": \"user\", \"content\": f\"Please think step by step and provide the most useful, detailed, thorough, and informative possible report. Please write each section and then put them all together to get somewhere around 1000 words if possible. Do not include the list of keywords found in the gap analysis individually, this takes up too much room. Keep in mind the domain you are giving recommendations for which is {user_domain}. the recommendations should make sense given this domain and not include recommendations that seem out of scope for the domain.\"}\n",
        "        ],\n",
        "        temperature=0.7,\n",
        "        max_tokens=3000,\n",
        "        top_p=1,\n",
        "        frequency_penalty=0,\n",
        "        presence_penalty=0\n",
        "    )\n",
        "\n",
        "    # Extract the generated report from the response\n",
        "    report = response[\"choices\"][0][\"message\"][\"content\"].strip()\n",
        "    report = report.strip()\n",
        "\n",
        "    # Save the report to a text file\n",
        "    with open(\"cluster_analysis_report.txt\", \"w\") as file:\n",
        "        file.write(report)\n",
        "\n",
        "    # Print the report\n",
        "    print(report)\n",
        "\n",
        "\n",
        "def truncate_keywords_to_tokens(keywords_list, max_tokens):\n",
        "    tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
        "    truncated_keywords = []\n",
        "    current_tokens = 0\n",
        "    for keyword in keywords_list:\n",
        "        keyword_tokens = tokenizer.tokenize(keyword)\n",
        "        if current_tokens + len(keyword_tokens) <= max_tokens:\n",
        "            truncated_keywords.append(keyword)\n",
        "            current_tokens += len(keyword_tokens)\n",
        "        else:\n",
        "            break\n",
        "    return truncated_keywords\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "user_domain = \"https://warbirdfishinggear.com/\"               #input(\"Please enter your website domain: \")\n",
        "\n",
        "topic_area = \"fishing gear\"             #input(\"Please enter the topic area you want to analyze \")\n",
        "\n",
        "min_difficulty = 0                      #input(\"Please enter the minimum keyword difficulty (0-100) \")\n",
        "\n",
        "max_difficulty = 100                    #input(\"Please enter the maximum keyword difficulty (0-100) \")\n",
        "\n",
        "num_websites = 4                        #input(\"Enter the number of competitor websites to consider \")\n",
        "\n",
        "num_keywords = 500                 #input(\"Enter the number of keywords to consider from each competitor\")\n",
        "\n",
        "user_position_range = (20, 100)         # Position range for the user's site\n",
        "\n",
        "competitor_position_range = (1, 20)     # Position range for the competitors' sites\n",
        "\n",
        "\n",
        "\n",
        "# Find the content gaps between your domain and your competitors domains\n",
        "content_gaps = identify_content_gaps(user_domain, topic_area, min_difficulty, max_difficulty, num_websites, num_keywords, user_position_range, competitor_position_range)\n",
        "print(content_gaps)\n",
        "\n",
        "# Convert the content_gaps set to a list\n",
        "content_gaps_list = list(content_gaps)\n",
        "\n",
        "# Truncate to make sure it can fit in the GPT context window\n",
        "max_tokens = 4400\n",
        "content_gaps_list = truncate_keywords_to_tokens(content_gaps_list, max_tokens)\n",
        "\n",
        "\n",
        "#Generate a content gap report authored by GPT\n",
        "generate_cluster_analysis_report(content_gaps_list, user_domain)\n",
        "\n",
        "# Generate embeddings for the keywords (use the appropriate function to generate embeddings)\n",
        "embeddings = generate_embeddings(content_gaps_list)\n",
        "\n",
        "# Perform clustering on the embeddings (use the appropriate function to perform clustering)\n",
        "cluster_assignments = perform_clustering(embeddings)\n",
        "\n",
        "# Create the DataFrame with keywords and cluster assignments\n",
        "clustered_keywords_df = create_clustered_keywords_df(content_gaps_list, cluster_assignments)\n",
        "\n",
        "# Get unique cluster IDs\n",
        "unique_cluster_ids = clustered_keywords_df['Cluster'].unique()\n",
        "\n",
        "# Generate descriptions for each cluster\n",
        "cluster_descriptions = describe_clusters(clustered_keywords_df)\n",
        "\n",
        "# Add cluster descriptions to the DataFrame for each keyword\n",
        "clustered_keywords_df['Cluster Description'] = clustered_keywords_df['Cluster'].map(cluster_descriptions)\n",
        "\n",
        "# Print the DataFrame\n",
        "print(clustered_keywords_df)\n",
        "\n",
        "# Save Clustered and Labeled dataset:\n",
        "clustered_keywords_df.to_csv(\"Content_Gap_Keywords_Clustered.csv\")\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clustered_keywords_df"
      ],
      "metadata": {
        "id": "1oFCQxWuS0wO"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}